diff --git a/drivers/serial/Makefile b/drivers/serial/Makefile
index 0e171b6..1dcc1c7 100644
--- a/drivers/serial/Makefile
+++ b/drivers/serial/Makefile
@@ -55,7 +55,10 @@
 COBJS-$(CONFIG_XILINX_UARTLITE) += serial_xuartlite.o
 COBJS-$(CONFIG_SCIF_CONSOLE) += serial_sh.o
 COBJS-$(CONFIG_TEGRA2) += serial_tegra2.o
+
+ifndef CONFIG_SPL_BUILD
 COBJS-$(CONFIG_USB_TTY) += usbtty.o
+endif
 
 COBJS	:= $(sort $(COBJS-y))
 SRCS	:= $(COBJS:.o=.c)